package com.gogant.spider.demo.image;

import com.gogant.spider.core.Context;
import com.gogant.spider.core.SpiderException;
import com.gogant.spider.http.HttpResponse;
import com.gogant.spider.http.HttpSpider;
import com.gogant.spider.http.HttpURLHandler;
import java.io.File;
import java.util.concurrent.TimeUnit;

public class ImageFetch {
    public static void main(String[] args) throws SpiderException {
        HttpSpider spider = new HttpSpider(5);
        spider.enableURLPicker()
                .when("com/$", "/home/\\d+$").find("/home/\\d+$", "/mm/\\d+$") // home
                .when("/mm/\\d+$").find("/mm/\\d+/\\d+$", "com/\\d+/\\d+/\\d+.jpg")//mm-page
                .when("/mm/\\d+/\\d+$").find("/mm/\\d+/\\d+$", "com/\\d+/\\d+/\\d+.jpg");// jpgs
        spider.addURLHandler(new HttpURLHandler() {
            File base = new File("e:/images");

            public boolean onFilter(Context context) {
                return context.like("%.jpg");
            }

            public void onResponse(HttpResponse resp) {
                System.out.println("fetch image: " + resp.getURL());
                resp.saveToPath(base);
            }

            public void onException(Context context, Throwable e) {
                System.err.println(e.getMessage());
            }
        });
        spider.fetch("http://www.mmjpg.com/", 100, 1, TimeUnit.DAYS);
        while (true) {
            System.out.println("remains: " + spider.getRemainURLCount() + ", workings: " + spider.getWoringURLCount());
            try {
                Thread.sleep(10000);
            } catch (InterruptedException e) {
            }
        }
    }
}
